Experiment Summary loader¶
In [2]:
from IPython.display import Javascript, display
from ipywidgets import widgets
def run_all(ev):
display(Javascript('IPython.notebook.execute_cells_below()'))
button = widgets.Button(description="Click to load experiment summary:")
button.on_click(run_all)
display(button)
In [3]:
# initializes dbconn and notebook settings
# imports relevant data from mysql db
import pandas as pd
import numpy as np
import sqlalchemy as sa
import glob
import os
import sys
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import datetime
%matplotlib inline
#%pylab inline
pylab.rcParams['figure.figsize'] = (15, 25)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import seaborn as sns
import qgrid
qgrid.nbinstall(overwrite=True)
qgrid.set_defaults(remote_js=True, show_toolbar=True)
import loaderplots as lp
from scipy.stats import stats, ttest_ind, ttest_rel, rankdata
#engine = sa.create_engine('mysql+pymysql://reeluser:R3el$olarX@localhost/rsi?charset=utf8')
engine = sa.create_engine('mysql+pymysql://reeluser:R3el$olarX@192.168.21.175/rsi?charset=utf8')
conn = engine.connect()
#cds, cells, exp, fullexp, fullprocess, ivt, ivtdfmod, mods, modtrends, process, sencera, trends, xrf
#senceradf = pd.read_sql("sencera", conn, index_col="sencera_index")
#processdf = pd.read_sql("process", conn, index_col="process_index")
#expdf = pd.read_sql("exp", conn, index_col="exp_index")
#celldf = pd.read_sql("cells", conn, index_col="cells_index")
#moddf = pd.read_sql("mods", conn, index_col="mods_index")
#%time trends = pd.read_sql("trends", conn, index_col="trends_index")
#xrf = pd.read_sql("xrf", conn, index_col="xrf_index")
#ivtdf = pd.read_sql("ivt", conn, index_col="ivt_index")
#eddf = pd.read_sql("ed", conn, index_col="ed_index")
import time
now = time.strftime("%c")
print ("Updated at: %s" % now )
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
To find experiments by topic:¶
- use the input box below to search by topic
- choose experiment ID from list
- input experiment ID in the box below the list
In [4]:
topic = (input("Enter experimental topic/description: "))
#cursor = conn.cursor()
#cursor.execute('SELECT DISTINCT `Experiment_ID`, `Description` FROM exp WHERE `Description` OR `Split1` OR `Split2` OR `Split3` LIKE "%{0}%";', (topic))
query = 'SELECT DISTINCT Experiment_ID, Description, Split1, Split2 FROM fullexp WHERE Description LIKE "%%{0}%%" OR Split1 LIKE "%%{0}%%" OR Split2 LIKE "%%{0}%%" OR Split3 LIKE "%%{0}%%";'
query = query.format(topic)
print ("\n")
print ("List of experiments involving %s updated at %s" % (topic, now) )
topicdf = pd.read_sql_query(query, conn)
qgrid.show_grid(topicdf, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
click "export" on the table to save it for offline use when exporting as html.
To load data and output summary, input Experiment ID below:¶
In [5]:
expt = int(input("Enter experiment number: "))
#query = 'SELECT * from trends WHERE `Experiment_ID` = %d;' % expt
query = 'SELECT * from filttrends WHERE `Experiment_ID` = %d;' % expt
df = pd.read_sql_query(query, conn)
print ("Imported Expt. %d at %s" % (expt, now) )
#'{0} in {1}'.format(unicode(self.author,'utf-8'),unicode(self.publication,'utf-8'))
In [6]:
#pd.set_option('display.notebook_repr_html', True)
#def _repr_latex_(self):
# return "\centering{%s}" % self.to_latex()
#pd.DataFrame._repr_latex_ = _repr_latex_ # monkey patch pandas DataFrame
module plots¶
Below, ranked module performance data.
In [7]:
#create module df
modsdf = df[df.Cell_Count > 19]
def mod_shunted(row):
if row['Rsh_Ohm'] <=500.:
return 1
modsdf.loc[:,'shunted'] = modsdf.apply(lambda row: mod_shunted(row), axis=1)
def mod_shunt(row):
if row['Rsh'] <=50.:
return 1
modsdf.loc[:,'shunt_count'] = modsdf.apply(lambda row: mod_shunt(row), axis=1)
#create nonshunted module df
nonshuntedmods = modsdf[modsdf.loc[:,'shunted'] != 1]
modsdflight = modsdf[~modsdf['Measurement'].str.contains('.*Dark.*', na = False)]
nonshuntedlight = nonshuntedmods[~nonshuntedmods['Measurement'].str.contains('.*Dark.*', na = False)]
#flatui1 = ["#3498db","#e74c3c", "#2ecc71", "#34495e", "#95a5a6", "#9b59b6", "#3498db", "#e74c3c",]
#sns.set_palette(flatui1)
#lp.mod_por_plots(modsdf, '%s' %expt)
query = 'SELECT Split1, Split2, Split3 from fullexp WHERE `Experiment_ID` = %s;' % expt
dfs = pd.read_sql_query(query, conn)
splitexist = []
splitexist = (dfs.isnull().all())
splitdict = dict(splitexist)
splitdict
def removeit(l):
d = dict(l)
for i in d.copy(): #if not d[i] returns true (null) splits
if d[i] == True:
d.pop(i)
return d
splitlist = removeit(splitexist)
modstrimmed = modsdflight.groupby('Sample_ID', as_index=False).max()
nonshuntedtrimmed = nonshuntedlight.groupby('Sample_ID', as_index=False).max()
In [8]:
lp.rankedfiltered(splitlist, modstrimmed, '%s' %expt)
here is just a simple chart showing shunted modules for POR vs EXP
In [9]:
#lp.shunts(splitlist, modstrimmed, '%s' %expt)
we can quickly output experimental stats
In [10]:
grouped = modsdflight.groupby('POR', as_index=True)
#exp is 0
modstats = pd.DataFrame(grouped.describe())
#modstats = modstats.set_index(['POR'])
print("Expt %d module stats: " % expt)
#modstats = modstats.drop('trends_index', axis=1)
modstats = modstats.drop('Experiment_ID', axis=1)
modstats = modstats.drop('Cell_Area_mm2', axis=1)
modstats = modstats.drop('Cell_Count', axis=1)
modstats = modstats.drop('Cell_no', axis=1)
modstats
#qgrid.show_grid(modstats, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
Out[10]:
performance by CdTe tool¶
In [11]:
plt.figure(figsize=(9,6))
ax = sns.stripplot(x="CdTe Tool", y="percentEff", data=modstrimmed, hue= "Split1", jitter=True)
sns.violinplot(x="CdTe Tool", y="percentEff", data=modstrimmed, inner=None, color=".8", ax=ax)
plt.legend(loc="lower left", frameon=True, fontsize=10)
Out[11]:
In [12]:
lp.bycdtetool(splitlist, modstrimmed, '%s' %expt)
performance vs deposition order¶
This loader takes user input. In the example below, you can choose to plot by Split1, Split2, POR, CdTe Tool, CdS Tool, or almost anything, vs. CdTe deposition time
In [13]:
lp.bydepplots(modstrimmed)
In [14]:
#lp.jointplots(modsdf, '%s' %expt)
cell plots¶
In [15]:
# create cells df
cellsdf = df[df.Cell_Count == 1]
def cell_shunted(row):
if row['Rsh_Ohm'] <=500.:
return 1
cellsdf['shunted'] = cellsdf.apply(lambda row: cell_shunted(row), axis=1)
def cell_shunt(row):
if row['Rsh'] <=50.:
return 1
cellsdf['shunt_count'] = cellsdf.apply(lambda row: cell_shunt(row), axis=1)
nscells = cellsdf[cellsdf.loc[:,'shunted'] != 1]
cellsdflight = cellsdf[~cellsdf['Measurement'].str.contains('.*Dark.*', na = False)]
nscellslight = nscells[~nscells['Measurement'].str.contains('.*Dark.*', na = False)]
cellgroup = cellsdflight.groupby('POR', as_index=False)
#nscellgroup = nscellslight.groupby('POR', as_index=False)
cellstrimmed = cellsdflight.groupby(['Sample_ID','Cell_no'], as_index=False).max()
nscellstrimmed = nscellslight.groupby(['Sample_ID','Cell_no'], as_index=False).max()
lp.rankedfiltered(splitlist, cellstrimmed, '%s' %expt)
nonshunted cell and module performance compared by Split¶
In [16]:
#nscells, cellsdflight, nscellslight, cellstrimmed, nscellstrimmed, cells df
#modsdf, modsdflight, modstrimmed,
#setting up grouped median and mean dfs
#modmedian = modsdf.groupby('Split1', as_index=False).median()
#modmean = modsdf.mean()
#cellmedian = cellsdf.groupby('Split1', as_index=False).median()
#cellmean = cellsdf.mean()
sns.set_palette("coolwarm")
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(figsize=(13,13), nrows=2, ncols=2)
#plt.subplot(411)
b = sns.boxplot('CdTe Tool', 'percentEff', hue='Split1', data=nonshuntedtrimmed, notch=False, ax=ax1)
# bplot = sns.boxplot('Sample_ID', 'percentEff', hue='POR', data=df, notch=False)
#plt.subplot(412)
b1 = sns.boxplot('CdTe Tool', 'percentEff', hue='Split1', data=nscellstrimmed, notch=False, ax=ax2)
#plt.subplot(421)
b2 = sns.boxplot('CdTe Tool', 'percentEff', hue='Split2', data=nonshuntedtrimmed, notch=False, ax=ax3)
#plt.subplot(422)
b3 = sns.boxplot('CdTe Tool', 'percentEff', hue='Split2', data=nscellstrimmed, notch=False, ax=ax4)
module data¶
In [23]:
qgrid.show_grid(modstrimmed, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
cell data¶
In [17]:
qgrid.show_grid(cellstrimmed, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
In [18]:
lp.expbycell(cellsdf, '%s' %expt)
Extras¶
Supplementary modules:
(1) T-test module: takes user input (POR, Splits, CdTe tool, etc)
(2) Past 50 experiments trends
(3) Module inspector: user selects module of interest; outputs Eff by cell (table and figure) and EL image(1) module t-testing¶
(1) input t-testing parameter when asked
(2) returns all permutations, sorted by significance
* typically uses modsdflight, but could use modstrimmed (may have higher errors)In [19]:
#the following function returns all ttests within a variable (ie Splits in Split 1, POR vs EXP, CDTE 1 vs 2)
#if instead you prefer to choose which samples to test, use: lp.ttestbychoice(df)
#df of choice is modsflight
lp.ttestall(modsdflight)
(2) Trends from most recent 50 experiments¶
In [20]:
flatui = ["#3498db", "#e74c3c", "#2ecc71", "#95a5a6", "#34495e", "#9b59b6"]
sns.set(style='ticks')
sns.set_palette(flatui)
sns.set_context("poster")
#uncomment for user input
#x_low = float(input("enter beginning of trends range: "))
#x_high = float(input("enter end of trends range: "))
#previous 50 experiments
x_low = int(df.Experiment_ID.unique() - 49.)
x_high = int(df.Experiment_ID.unique()) + 1
query = 'SELECT * FROM filttrends WHERE Experiment_ID BETWEEN {0} AND {1} AND Measurement LIKE "Light" AND Cell_Count > 20 ;'
query = query.format(x_low, x_high)
trendsdf = pd.read_sql_query(query, conn)
lp.portrends(trendsdf)
(3) sample inspection¶
(1) use the table below to choose module
(2) input Sample ID when askedmodule data¶
In [21]:
samps = []
query = 'SELECT DISTINCT `Sample_ID`, Description, percentEff, Split1, Split2, Split3 from filttrends WHERE `Cell_Count` > 20 AND `Experiment_ID` = %d AND Measurement LIKE "Light";' % expt
samps = pd.read_sql_query(query, conn)
samplist = samps["Sample_ID"].tolist()
samps2 = samps.groupby('Sample_ID', as_index=False).max()
qgrid.show_grid(samps2, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
In [22]:
sample = (input("Enter sample ID: ")).upper()
#cursor = conn.cursor()
#cursor.execute('SELECT DISTINCT `Experiment_ID`, `Description` FROM exp WHERE `Description` OR `Split1` OR `Split2` OR `Split3` LIKE "%{0}%";', (topic))
query = 'SELECT * FROM filttrends WHERE Sample_ID LIKE "%%{0}%%" AND Cell_Count = 1 AND Measurement LIKE "Light";'
query = query.format(sample)
sampledf = pd.read_sql_query(query, conn)
qgrid.show_grid(sampledf, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 200})
sns.set_style("darkgrid")
fg = sns.FacetGrid(sampledf, hue='POR', size= 6, aspect=2, )#, hue_kws=dict(marker=["^", "v"]))
fg.map(plt.scatter, 'Cell_no', 'percentEff', alpha=1,s=50)
#plt.xlim(570,700)
plt.ylim(0, 18)
plt.axhline(16.7, color='r', linestyle='-.')
pins = [1, 4, 7, 10, 13, 16, 19, 22]
for xc in pins:
plt.axvline(x=xc, color = 'c', linestyle = ':', lw=2)
#plt.axvline(1, color='r', linestyle='-.')
plt.grid(b=True, which='both', color='0.65',linestyle='-', )
#plt.legend()
plt.tick_params(axis='both',which='minor')
plt.minorticks_on()
plt.title('%s eff by cell' % sample)
eldate = str(sampledf["Date_tm"].unique())
eldate = eldate[2:12]
#eldate = eldate[7:11]+"-0"+eldate[2:3]+"-"+eldate[4:6]
#print(eldate)
from IPython.display import Image
PATH = '//rsi-el/Users/Public/EL/'
#filename =PATH + "%s/%s.jpg" % (eldate,sample)
#print(filename)
Image(filename = PATH + "%s/%s.jpg" % (eldate,sample), width=700, height=700)
Out[22]:
In [ ]: